<?php
namespace app\admin\controller;
use think\Db;
use think\Config;
use phpspider\core\phpspider;
use phpspider\core\requests;
use phpspider\core\selector;
class Index
{
    public function index()
    {
        $name = request()->param('name');
        $a = config();
        dump($a);
        return json(array('status'=>1,'msg'=>'你好'.$name));
    }

    public function add(){
        $data['id'] =1;
        $data['name'] =1;
        $data['remark'] =1;
        $data['sort'] =1;
       $list =  Db::name('test')->insert($data);
       dump($list);
    }

    public function spider(){
        $configs = array(
            'name' => '第一个爬虫测试',
            'log_show' => true,
            'fields' => array(
                // 标题
                array(
                    'name' => "name",
                    'selector' => "//title",
                    'required' => true,
                ),
                array(
                    'name' => 'content',
                    'selector' => "//div[@class='p']"
                )
            ),
            'user_agent' => array(
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36",
                "Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_3 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13G34 Safari/601.1",
                "Mozilla/5.0 (Linux; U; Android 6.0.1;zh_cn; Le X820 Build/FEXCNFN5801507014S) AppleWebKit/537.36 (KHTML, like Gecko)Version/4.0 Chrome/49.0.0.0 Mobile Safari/537.36 EUI Browser/5.8.015S",
            ),
            'export' => array(
                'type' => 'csv',
                'file' => './data/qiushibaike.html', // data目录下
            ),
            'domains' => array(
                'www.17k.com',
                '17k.com'
            ),
            'scan_urls' => array(
                "https://www.17k.com/chapter/2921783/36488048.html",
            ),
            'content_url_regexes' => array(
                "https://www.17k.com/chapter/2921783/\d+.html",
            ),

        );


        $spider = new phpspider($configs);

        $spider->on_start = function($phpspider)
        {
            for ($i = 2; $i <= 932; $i++)
            {
                $url = "http://m.52mnw.cn/ikaimi/morepic.php?classid=6,7,8,10,11,15&line=10&order=newstime&page={$i}";
                $phpspider->add_scan_url($url);
            }
        };

        $spider->on_start = function($phpspider)
        {
            requests::set_header("Referer", "http://buluo.qq.com/p/index.html");
        };


        $spider->on_extract_field = function($fieldname, $data, $page)
        {
            if ($fieldname == 'content')
            {
                $content = preg_replace("/<div(([\s\S])*?)<\/div>/","",$data);
                $content = preg_replace("/\t/","",$content);
                $content = selector::remove($content,"//p[contains(@class,'copy')]");
            }
            return $content;
        };

        $spider->is_anti_spider = function($url, $content, $phpspider)
        {
            // $content中包含"404页面不存在"字符串
            if (strpos($content, "404页面不存在") !== false)
            {
                // 如果使用了代理IP，IP切换需要时间，这里可以添加到队列等下次换了IP再抓取
                // $phpspider->add_url($url);
                return true; // 告诉框架网页被反爬虫了，不要继续处理它
            }
            // 当前页面没有被反爬虫，可以继续处理
            return false;
        };
        $spider->on_download_attached_page = function($content, $phpspider)
        {
            $content = trim($content);
            $content = preg_replace("/[\s]{2,}/","",$content);
            //    echo $content;
            $content = json_decode($content, true);
            return $content;
        };

        $spider->start();
    }
}
